import math
from collections import defaultdict

import document_query_processing

class Comparison:
    def __init__(self, dqp):
        self.inv_dict = defaultdict(list)
        self.query_list = []
        self.term_list = []
        self.term_list_size = 0
        self.total_term = 0
        self.result = []
        self.dqp = dqp
    
    def is_number(self, s):
        try:
            float(s)
            return True
        except ValueError:
            return False
    
    """searchFileQuery(string,string)"""
    def searchFileQuery(self, inverted_file_name, query_file_name):
        inverted_file = open(inverted_file_name, 'r')
        query_file = open(query_file_name, 'r')
        
        s = []
        inverted_file.readline() # header skip
        for line in inverted_file.readlines():
            kata = ""
            stat = []
            for word in line.split(' '):
                if(self.is_number(word) == False):
                    if kata != "":
                        kata += " "
                    kata += word
                else:
                    stat.append(float(word))
            docid = stat[0]
            rawtf = stat[1]
            df = stat[2]
            weight = stat[3]
            s.append((kata, (int(docid),float(rawtf),float(df),float(weight)) ))
        
        self.inv_dict = defaultdict(list)
        for k,v in s:
            self.inv_dict[k].append(v)
        
        self.total_term = len(self.inv_dict)
        
        self.term_list = []
        for x in self.inv_dict:
            #print x,self.inv_dict[x]
            if x not in self.term_list:
                self.term_list.append(x)
        self.term_list_size = len(self.term_list)
            
        query = ""
        self.query_list = []
        for line in query_file.readlines():
            if line[0] == '.': # not query
                if query != "":
                    self.query_list.append(self.dqp.queryProcessing(query))
                    query = ""
            else: # query
                query += line
        self.query_list.append(self.dqp.queryProcessing(query))
        #print self.query_list
            
    """searchOneQuery(string,string,string)"""       
    def searchOneQuery(self, inverted_file_name, query_string):
        inverted_file = open(inverted_file_name, 'r')
        
        s = []
        inverted_file.readline() # header skip
        for line in inverted_file.readlines():
            kata = ""
            stat = []
            for word in line.split(' '):
                if(self.is_number(word) == False):
                    if kata != "":
                        kata += " "
                    kata += word
                else:
                    stat.append(float(word))
            docid = stat[0]
            rawtf = stat[1]
            df = stat[2]
            weight = stat[3]
            s.append((kata, (int(docid),float(rawtf),float(df),float(weight)) ))
        
        self.inv_dict = defaultdict(list)
        for k,v in s:
            self.inv_dict[k].append(v)
        
        self.total_term = len(self.inv_dict)
        
        self.term_list = []
        for x in self.inv_dict:
            #print x,self.inv_dict[x]
            if x not in self.term_list:
                self.term_list.append(x)
        self.term_list_size = len(self.term_list)
            
        query = query_string
        self.query_list = []
        self.query_list.append(self.dqp.queryProcessing(query))
       # print self.query_list
        
    def calcDocumentVector(self):
        # return list of document vector
       # print self.dqp.total_document
        ret = [0]*self.dqp.total_document
        #print self.term_list_size
        for i in range(len(ret)):
            ret[i]=[0]*self.term_list_size
        
        idx = 0
        #print ret
        for term in self.inv_dict:
            for termstat in self.inv_dict[term]:
                # ret[docid][term] = weight
                ret[termstat[0]-1][idx] = termstat[3]
            idx += 1
        #print "Banyak document =", len(ret)
        #print "Document Vector =", ret
        return ret
    
    def vectorSize(self,vector):
        # return size of a vector
        size = 0
        for x in vector:
            size += x * x
        return math.sqrt(size)
            
    def calcQueryVector(self, query_code):
        # return query vector
        # ambil tf dari query, df dari dokumen
        tf_code = query_code[0]
        idf_code = query_code[1]
        norm_code = query_code[2]
        
        ret = [0]*len(self.query_list)
        for i in range(len(ret)):
            ret[i]=[0]*self.term_list_size
        #idx = 0
        #for query in self.query_list:
            #for term in query:
                #if term in self.inv_dict:
                    #for termstat in self.inv_dict[term]:
                        #print termstat[0]
                        #ret[idx][term_list.index(term)] = termstat[4]
        idx = 0
        tf_max = [0]*len(self.query_list) # max tf for each query
        for query_dict in self.query_list:
            for query in query_dict:
                if query_dict[query] >= tf_max[idx]:
                    tf_max[idx] = query_dict[query]                        
            idx += 1
        
        idx = 0
        #print "tf_max =",tf_max
        #print "QUERY_LIST = ",self.query_list                        
        for query in self.query_list:
            #print "Query=",query
            for term in query:
                # tf x idf  = query[term] * idf
                #print "Term =", term
                #print "TERM LIST = ", self.term_list
                if term not in self.term_list:
                    continue
                termid = self.term_list.index(term)
                tf = query[term]               
                if tf_code == "n":
                    pass
                elif tf_code == "l":
                    tf = 1 + math.log10(tf)
                elif tf_code == "b":
                    if tf > 0:
                        tf = 1
                    else:
                        tf = 0
                elif tf_code == "a":
                    return 0.5 + 0.5*(tf*1.0/tf_max[idx])

                ret[idx][termid] = tf
                #print "TERMSTAT"
                #print "INV_DICT",self.inv_dict
                for termstat in self.inv_dict[term]:
                    #print termstat
                    #print termstat[0], termid
                    #if termstat[0] == termid:
                    #print "masuk"
                    if idf_code == "n":
                        ret[idx][termid] *= 1
                    elif idf_code == "t":
                        if termstat[2] > 0:
                            ret[idx][termid] = math.log10(self.dqp.total_document/termstat[2])
                        else:
                            ret[idx][termid] = math.log10(self.dqp.total_document)
            idx += 1
        
        #print "RET (before norm) = ", ret
        if norm_code == "c":
             for i in range(len(ret)):
                normalizer = self.vectorSize(ret[i])
                #print normalizer
                for j in range(len(ret[i])):
                    if normalizer==0:
                        continue
                    ret[i][j] /= normalizer
                    
        #print "RET (after norm) = ", ret
       # print ret
       # print "Query vector=", ret
        return ret
            
    """calcSimilarity(list<int>,list<int>)"""
    def calcSimilarity(self, query, document):
        # Sim(Q,D) = Q.D/(|Q|.|D|)
        sim = 0
        
        for i in range(len(query)):
            sim += query[i] * document[i]
        
        return sim
    
    def calcSimiDict(self, query_dict, document):
        sim = 0
        
        #print self.term_list
        for term in query_dict:
            sim += query_dict[term] * document[self.term_list.index(term)]
            
        return sim
            
    def calcRank(self, query, documents, threshold): 
        # return list of rank of documents relevancy to one query, sorted descending by weight
        ret = []
        docsize = len(documents)
#        print "DOCUMENTS = ", documents
        for i in range(docsize):
            sim = self.calcSimilarity(query, documents[i])
            if sim < threshold:
                continue
            ret.append((i+1, sim))
            
        #print ret
        ret = sorted(ret, key = lambda ret: ret[1]) # sort ascending
        ret = [x[0] for x in ret]
        ret.reverse()
        #print ret
        return ret
    
    def reCalcAllRank(self, q1, documents, threshold):
        ret = []
        docsize = len(documents)
        
        for i in range(docsize):
            sim = self.calcSimiDict(q1, documents[i])
            if sim < threshold:
                continue
            ret.append((i+1, sim))
        
        ret = sorted(ret, key = lambda ret: ret[1]) # sort ascending
        ret = [x[0] for x in ret]
        ret.reverse()
        #print ret
        return ret
    
    def calcAllRank(self, queries, documents, threshold):
        ret = []
        querysize = len(queries)
        for i in range(querysize):
            ret.append((i+1, self.calcRank(queries[i], documents, threshold)))
        self.result = ret
        return ret
    
    def writeRetrievalResult(self, file_name):
        # write relevant result, sorted descending by rank
        f = open(file_name, 'w')
        #print self.result
        for res in self.result:
            for docid in res[1]:
                f.writelines(str(res[0]) + " " + str(docid) + "\n")
    
    def createWeightDict(self, query):
        # list of list query vector [[1,1,1,0,0,0],[1,2,2]]
        # dict (term:weight)
        qv = self.calcQueryVector(self.dqp.query_term_weighting_code)
        ret = {}
        j = 0
        query_term_dict = self.dqp.queryProcessing(query)
        for i in self.term_list:
            if qv[0][j] == 0:
                if i in query_term_dict:
                    ret[i] = 0
                else:
                    pass
            else:
                ret[i] = qv[0][j]
            j += 1

        #print ret 
        return ret
     
if __name__ == '__main__':
    #document_collection_file = 'simple_document_collection.txt'
    #document_collection_file = 'document_collection.txt'
    document_collection_file = 'simple_document_collection.txt'
    
    query_list_file = 'query_list.txt'
    stop_list_file = 'stoplist.txt'
    term_weighting_code = 'lnc.ltc'
    inverted_file_saved_path = 'inverted_file_document_collection.txt'
    
    is_use_stemming = True
    qproc = document_query_processing.DocumentQueryProcessing(document_collection_file, stop_list_file, is_use_stemming, term_weighting_code, inverted_file_saved_path)
    qproc.documentCollectionProcessing()
    
    comp = Comparison(qproc)
    #comp.searchFileQuery('inverted_file_document_collection.txt', 'simple_query_list.txt')
    comp.searchOneQuery('testcase/inverted_file.txt', 'red big car')
    #comp.calcAllRank(comp.calcQueryVector("ltc"), comp.calcDocumentVector(), 0.0)
    comp.writeRetrievalResult('retrieval_result.txt')
    #comp.createWeightDict('red big car')
    q1 = {'red':0.7,'big':0.8,'car':0.4}
    comp.reCalcAllRank(q1, comp.calcDocumentVector(), 0.0)
    
